#include "stdafx.h"
#include "memes.h"
#include <time.h>
//#include <direct.h>

THash<TChA, TInt> StatsH;


void BigMain(int argc, char* argv[]);

void ConstrainRange(TIntH&, int, int);
double ComputeSim(THashSet<TMd5Sig>& FirstSet, THashSet<TMd5Sig>& LastSet);
void FGetFileList(TStr InFNm, TChAV& filelist) {
  TChA Ln;

	// Get filelist from file
	PSIn InFNmF = TFIn::New(InFNm);
	TStr FNm;
	while (! InFNmF->Eof() && InFNmF->GetNextLn(FNm))
		filelist.Add(FNm);

	TChAV DelList; 
	// Test whether each file is good
	for (int i = 0; i < filelist.Len(); i++) {
		FILE *fin = fopen(filelist[i].CStr(), "r");
		if (fin == NULL) {
			printf("Error reading file %s, ignore...\n", filelist[i].CStr());
			DelList.Add(filelist[i]);
		} else {
			fclose(fin);
		}
	}
	for (int i = 0; i < DelList.Len(); i++)
		filelist.DelIfIn(DelList[i]);

}
void FQuoteFilter(TStr &q)
{
	TStrV words;	
	q.SplitOnAllAnyCh(" ?!()@#=&,.<>/\\:\";{}|", words);	
	q.Clr();
	for (int i=0; i<words.Len(); ++i)
	{
		if (i>0)
			q.InsStr(q.Len()," ");
		q.InsStr(q.Len(),words[i]);
	}
}


int main(int argc, char* argv[]) {
  printf("MemeWorks. build: %s, %s. Start time: %s\n\n", __TIME__, __DATE__, TExeTm::GetCurTm());
  //TSysProc::SetLowPriority();
  TExeTm ExeTm;  TInt::Rnd.PutSeed(0);  Try
  //clock_t BegClk = clock(), EndClk;
  TSecTm BegTm = TSecTm::GetCurTm();

// Meme to Quote Base
//  char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:test.txt",  "-o:test", "-mint:20110101", "-maxt:20110601"};  BigMain(6, ToDo);   
//	char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:201102.txt",  "-o:1102", "-f:7", "-mint:20110201", "-maxt:20110301"};  BigMain(7, ToDo);   
//	char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:201108.txt",  "-o:1108", "-f:7", "-mint:20110801", "-maxt:20110901"};  BigMain(7, ToDo);   
//		char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:20110608.txt",  "-o:110608", "-f:7", "-mint:20110601", "-maxt:20110901"};  BigMain(7, ToDo);   
//  char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:201007_201107.txt", "-o:10071107", "-mint:20100714", "-maxt:20110728"};  BigMain(6, ToDo); 
//	  char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:201101_201107.txt", "-o:11011107", "-mint:20110101", "-maxt:20110801"};  BigMain(6, ToDo);
//	char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:201108.txt",  "-o:test1108", "-f:5", "-mint:20110801", "-maxt:20110805"};  BigMain(7, ToDo);
//		char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:20110409.txt",  "-o:110409", "-f:7", "-mint:20110401", "-maxt:20111001"};  BigMain(7, ToDo);

// Meme Clustering
//  char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:110608-w4mfq7.QtBs", "-mf:5", "-o:110608", "-shglready:T", "-netready:T"};  BigMain(7, ToDo);  
//	char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:1102-w4mfq7.QtBs", "-o:1102", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);  
//	char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:1108-w4mfq7.QtBs", "-mf:5", "-o:1108", "-shglready:F", "-netready:F"};  BigMain(7, ToDo);  
//	char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:test1108-w4mfq5.QtBs", "-mf:5", "-cf:5", "-o:1108", "-shglready:F", "-netready:F"};  BigMain(8, ToDo);  

//	char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:1101-w4mfq5.QtBs", "-o:1101", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);  
//  char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:qt0808-w4mfq5.QtBs", "-o:0808", "-shglready:T", "-netready:T"};  BigMain(6, ToDo);  
//  char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:11011103-w4mfq5.QtBs", "-o:11011103", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);  
//	char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:11041106-w4mfq5.QtBs", "-o:11041106", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);  
//  char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:10071107-w4mfq5.QtBs", "-o:10071107", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);  
//	char *ToDo [] = {"QuotesApp", "-do:inspect", "-i:201108.txt",  "-o:1108", "-mint:20110801", "-maxt:20110826"};  BigMain(6, ToDo);

// Sample Quote Base
//	  char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:110608-w4mfq30.QtBs", "-o:110608", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);
		char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:110608-Cluster.QtBs", "-o:110608", "-shglready:F", "-netready:F"};  BigMain(6, ToDo);
//	char *ToDo [] = {"QuotesCluster", "-do:mkclustnet", "-i:1108-w4mfq15.QtBs", "-o:1108", "-shglready:T", "-netready:T"};  BigMain(6, ToDo);  


  TSecTm EndTm = TSecTm::GetCurTm();
  double usedTime = EndTm.GetAbsSecs() - BegTm.GetAbsSecs();
  printf("Total execution time : %02dh%02dm%02ds\n", int(usedTime)/3600, (int(usedTime)%3600)/60, int(usedTime)%60);
  return 0;
  
  CatchFull
  printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
  return 0;
}

void BigMain(int argc, char* argv[]) {
  TExeTm ExeTm;
  Env = TEnv(argc, argv, TNotify::StdNotify);
  Env.PrepArgs("QuotesApp");
  const TStr ToDo = Env.GetIfArgPrefixStr("-do:", "", "To do").GetLc();
  if (Env.IsEndOfRun()) {
    printf("To do:\n");
    printf("    MkDataset         : Make memes dataset (extract quotes and save txt)\n");
    printf("    ExtractSubset     : Extract a subset of memes containing particular words\n");
    printf("    MemesToQtBs       : Load memes dataset and create quote base\n");
    printf("    MkClustNet        : Build cluster network from the quote base\n");
    return;
  }	
#pragma region mkdataset
  // extract quotes and links and make them into a single file
  if (ToDo == "mkdataset") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "files.txt", "Spinn3r input files (one file per line)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "Spinn3r-dataset.txt", "Output file");
    const int MinQtWrdLen = Env.GetIfArgPrefixInt("-w:", 3, "Minimum quote word length");
    const TStr UrlFNm = Env.GetIfArgPrefixStr("-u:", "", "Seen url set (THashSet<TMd5Sig>) file name");
    const bool UrlOnlyOnce = Env.GetIfArgPrefixBool("-q:", true, "Only keep unique Urls");
    //// parse directly from Spinn3r
    TStr Spinn3rFNm;
    THashSet<TMd5Sig> SeenUrlSet;
    if (UrlOnlyOnce && ! UrlFNm.Empty()) {  // keep track of already seen urls (so that there are no duplicate urls)
      TFIn FIn(UrlFNm);  SeenUrlSet.Load(FIn);
    }
    FILE *F = fopen(OutFNm.CStr(), "wt");
    TFIn FIn(InFNm);
    int Items=0;
    for (int f=0; FIn.GetNextLn(Spinn3rFNm); f++) {
      TQuoteExtractor QE(Spinn3rFNm.ToTrunc());
      printf("Processing %02d: %s [%s]\n", f+1, Spinn3rFNm.CStr(), TExeTm::GetCurTm());
      fflush(stdout);
      for (int item = 0; QE.Next(); item++) {
        const TMd5Sig PostMd5(QE.PostUrlStr);
        if (QE.QuoteV.Empty() && QE.LinkV.Empty()) { continue; } // no quotes, no links
        if (UrlOnlyOnce) {
          if (SeenUrlSet.IsKey(PostMd5)) { continue; }
          SeenUrlSet.AddKey(PostMd5);
        }
        fprintf(F, "P\t%s\n", QE.PostUrlStr.CStr());
        //if (QE.PubTm > TSecTm(2008,8,30) || QE.PubTm < TSecTm(2008,7,25)) { printf("%s\n", QE.PubTm.GetStr().CStr()); }
        fprintf(F, "T\t%s\n", QE.PubTm.GetYmdTmStr().CStr());
        for (int q = 0; q < QE.QuoteV.Len(); q++) {
          if (TStrUtil::CountWords(QE.QuoteV[q]) >= MinQtWrdLen) {
            fprintf(F, "Q\t%s\n", QE.QuoteV[q].CStr()); }
        }
        for (int l = 0; l < QE.LinkV.Len(); l++) {
          fprintf(F, "L\t%s\n", QE.LinkV[l].CStr()); }
        fprintf(F, "\n");
        if (item>0 && item % Kilo(100) == 0) {
          QE.DumpStat();  QE.ExeTm.Tick(); }
        Items++;
      }
      printf("file done. Total %d all posts, %d all items\n", SeenUrlSet.Len(), Items);
      fflush(stdout);
    }
    printf("all done. Saving %d post urls\n", SeenUrlSet.Len());  fflush(stdout);
    if (! SeenUrlSet.Empty()) {
      TFOut FOut(OutFNm.GetFMid()+".SeenUrlSet");
      SeenUrlSet.Save(FOut);
    }
    fclose(F);
  }
#pragma endregion mkdataset

#pragma region extractsubset
  // save posts with memes containing particular words
  else if (ToDo == "extractsubset") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "memes_*.rar", "Input file prefix");
    const bool IsInFNmWc = Env.GetIfArgPrefixBool("-w:", true, "Input is wildcard (else a file with list of input files)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "memes-subset.txt", "Output memes file");
    const TStr WordsFNm = Env.GetIfArgPrefixStr("-p:", "phrases-in.txt", "Phrases that memes have to contain");

    TChAV CatchMemeV;// = TStr::GetV("great depression", "economic meltdown", "recession had bottomed out", "green shoots", "slow recovery", "gradual recovery");
    printf("Loading %s\n", WordsFNm.CStr());
    { TFIn FIn(WordsFNm);
    for (TStr Ln; FIn.GetNextLn(Ln); ) {
      printf("  %s\n", Ln.GetLc().CStr());
      CatchMemeV.Add(Ln.GetLc()); }
    }
    printf("%d strings loaded\n", CatchMemeV.Len());
    TFOut FOut(OutFNm);
    TMemesDataLoader Memes(InFNmWc, IsInFNmWc);
    for (int posts = 0, nsave=0; Memes.LoadNext(); posts++) {
      bool DoSave = false;
      for (int m = 0; m < Memes.MemeV.Len(); m++) {
        for (int i = 0; i < CatchMemeV.Len(); i++) {
          if (Memes.MemeV[m].SearchStr(CatchMemeV[i]) != -1) {
            DoSave=true; break; }
        }
        if (DoSave) { break; }
      }
      if (DoSave) { Memes.SaveTxt(FOut); nsave++; }
      if (posts % Mega(1) == 0) {
        printf("%dm posts, %d saved\n", posts/Mega(1), nsave);
        FOut.Flush();
      }
    }
  }
#pragma endregion extractsubset

#pragma region memestoqtbs
  // load memes dataset (MkDataset) and create quote base
  else if (ToDo == "memestoqtbs") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "201007_201107.txt", "Input Memes dataset files");
    const TStr MediaUrlFNm = Env.GetIfArgPrefixStr("-u:", "news_media.txt", "Fule with news media urls");
    const TStr Pref = Env.GetIfArgPrefixStr("-o:", "qt", "Output quote base");
    const int MinWrdLen = Env.GetIfArgPrefixInt("-l:", 4, "Min quote word length");
    const int MinMemeFq = Env.GetIfArgPrefixInt("-f:", 5, "Min meme frequency");
		const TStr MinTmStr = Env.GetIfArgPrefixStr("-mint:", "20100714", "Min time of quotes, format = YYYYMMDD");
		const TStr MaxTmStr = Env.GetIfArgPrefixStr("-maxt:", "20110728", "Max time of quotes, format = YYYYMMDD");
		TSecTm MinTm(atoi(MinTmStr.GetSubStr(0,3).CStr()),atoi(MinTmStr.GetSubStr(4,5).CStr()),atoi(MinTmStr.GetSubStr(6,7).CStr()));
		TSecTm MaxTm(atoi(MaxTmStr.GetSubStr(0,3).CStr()),atoi(MaxTmStr.GetSubStr(4,5).CStr()),atoi(MaxTmStr.GetSubStr(6,7).CStr()));

		TmYMDH::GenSpinn3rList(TmYMDH(MinTm), TmYMDH(MaxTm), TStr("../../spinn3r-full3/"), InFNm);

		PQuoteBs QtBs = TQuoteBs::New();
		int HashTableSize = 200;
		int UrlSetSize = 4 * HashTableSize;
		QtBs->ConstructQtBsZarya(InFNm, Pref, MediaUrlFNm, MinTm, MaxTm, MinWrdLen, MinMemeFq, HashTableSize, UrlSetSize);
  }
#pragma endregion memestoqtbs


#pragma region mkclustnet
  // make cluster network
  else if (ToDo == "mkclustnet") {
    TStr InQtBsNm = Env.GetIfArgPrefixStr("-i:", "", "Input quote base file name");
    TStr Pref = Env.GetIfArgPrefixStr("-o:", "qt", "Output network/updated QtBs filename");
		TStr BlackListFNm = Env.GetIfArgPrefixStr("-b:", "quote_blacklist.txt", "Blacklist file name");
		bool IsShglReady = Env.GetIfArgPrefixBool("-shglready:", false, "Indicate whether shingle hash table is ready");
		bool IsNetReady = Env.GetIfArgPrefixBool("-netready:", false, "Indicate whether cluster net is ready");
		double BktThresh = Env.GetIfArgPrefixFlt("-bktthresh:", 0.4, "Threshold for bad shingle bucket elimination");
		double MxTmDelay = Env.GetIfArgPrefixFlt("-delaythresh:", 5, "Max time delay between two quotes in the same cluster");
		double MxTmDev = Env.GetIfArgPrefixFlt("-devthresh:", 3, "Max time deviation for a quote to be specific rather than general");
		double RefineThresh = Env.GetIfArgPrefixFlt("-refinethresh:", 0.2, "Threshold for merging quote cluster in refining process");
		const int MinWrdLen = Env.GetIfArgPrefixInt("-minl:", 4, "Min quote word length");
		const int MaxWrdLen = Env.GetIfArgPrefixInt("-maxl:", 200, "Max quote word length");
		const int MinMemeFq = Env.GetIfArgPrefixInt("-mf:", 5, "Min meme frequency");
		const int MinClustFq = Env.GetIfArgPrefixInt("-cf:", 50, "Min quote cluster frequency");

		//printf("Make dir\n");
		//mkdir(".\\Data\\", S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
		//system("mkdir -p ./Data/Data/");
		//printf("DONE");
		//FILE* F = fopen("./Data/ouput.txt", "w");
		//fprintf(F, "Hello\n");
		//fclose(F);

		// Load quote base
    PQuoteBs QtBs;
    if (TZipIn::IsZipFNm(InQtBsNm)) { TZipIn ZipIn(InQtBsNm);  QtBs = TQuoteBs::Load(ZipIn); }
    else { TFIn FIn(InQtBsNm);  QtBs = TQuoteBs::Load(FIn); }
		
		// Cluster the quotes
    QtBs->ClusterQuotes(MinMemeFq, MinWrdLen, MaxWrdLen, BlackListFNm, Pref, IsShglReady, IsNetReady, BktThresh, MxTmDelay, MxTmDev, RefineThresh);
		//QtBs->RefineQtBs(BlackListFNm, MinMemeFq, MinWrdLen, MaxWrdLen);

		// Quote trend analysis
		QtBs->AnalyzeQuoteTrend(Pref, tmu1Hour, 5);
//		QtBs->GetSampleQtBs(10, "110608");

		// Dump the clusters
		//bool SkipUrl = true, FlashDisp = true;
		//QtBs->DumpQuoteClusters(MinWrdLen, MinClustFq, SkipUrl, FlashDisp, Pref);

		// Compute the Domain Net
//		TNodeEDatNet<TInt, TInt> DomNet;
//		THash<TMd5Sig, TInt> DomIdH;
//		THash<TInt, TChA> DomStrH;
//#if DomGraphReady
//		TFIn FIn(Pref+"-DomGraph.bin"); DomNet = *(TNodeEDatNet<TInt, TInt>::Load(FIn)); DomIdH.Load(FIn), DomStrH.Load(FIn);
//#else
//		QtBs->GetDomGraph(DomNet, DomIdH, DomStrH); TFOut FOut(Pref+"-DomGraph.bin"); DomNet.Save(FOut); DomIdH.Save(FOut); DomStrH.Save(FOut);
//#endif
//		QtBs->Validate(0, 1, DomNet, DomIdH, DomStrH);
//		QtBs->ProcessEdge(DomNet, DomIdH, DomStrH);
  }
#pragma endregion mkclustnet

#pragma region inspect
  // Inspect the property of the urls
  else if (ToDo == "inspect") {
		const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "201007_201107.txt", "Input Memes dataset files");
		const TStr MediaUrlFNm = Env.GetIfArgPrefixStr("-u:", "news_media.txt", "Fule with news media urls");
		const TStr Pref = Env.GetIfArgPrefixStr("-o:", "qt", "Output quote base");
		const int MinWrdLen = Env.GetIfArgPrefixInt("-l:", 4, "Min quote word length");
		const int MinMemeFq = Env.GetIfArgPrefixInt("-f:", 5, "Min meme frequency");
		const TStr MinTmStr = Env.GetIfArgPrefixStr("-mint:", "20100714", "Min time of quotes, format = YYYYMMDD");
		const TStr MaxTmStr = Env.GetIfArgPrefixStr("-maxt:", "20110728", "Max time of quotes, format = YYYYMMDD");
		TSecTm MinTm(atoi(MinTmStr.GetSubStr(0,3).CStr()),atoi(MinTmStr.GetSubStr(4,5).CStr()),atoi(MinTmStr.GetSubStr(6,7).CStr()));
		TSecTm MaxTm(atoi(MaxTmStr.GetSubStr(0,3).CStr()),atoi(MaxTmStr.GetSubStr(4,5).CStr()),atoi(MaxTmStr.GetSubStr(6,7).CStr()));

		TmYMDH::GenSpinn3rList(TmYMDH(MinTm), TmYMDH(MaxTm), TStr("../../spinn3r-full3/"), InFNm);
		PQuoteBs QtBs = TQuoteBs::New();

		TChAV filelist;
		FGetFileList(InFNm, filelist);

		int UrlSize = Mega(200);
		int RecSize = Mega(100);

		int fileCnt = 0;
		THash<TMd5Sig, TSecTm> FirstTmH(UrlSize, true), LastTmH(UrlSize, true);
		#pragma omp parallel for schedule(dynamic)
		for (int i = 0; i < filelist.Len(); i++) {
			TMemesDataLoader Memes;
			TMd5SigV UrlSigV;
			TVec<TSecTm> TmV;

			#pragma omp critical
			Memes.LoadFile(filelist[i]);
			while (Memes.LoadNextSkip()) {
				if (strstr(Memes.PostUrlStr.CStr(), "facebook.com") != NULL) continue;
				if (strstr(Memes.PostUrlStr.CStr(), "twitter.com") != NULL) continue;
				UrlSigV.Add(TMd5Sig(Memes.PostUrlStr));
				TmV.Add(Memes.PubTm);
			}

			#pragma omp critical 
			{
				for (int j = 0; j < UrlSigV.Len(); j++) {
					TMd5Sig& UrlSig = UrlSigV[j];
					TSecTm& Tm = TmV[j];
					if (!FirstTmH.IsKey(UrlSig) || FirstTmH.GetDat(UrlSig) > Tm)
						FirstTmH.AddDat(UrlSig) = Tm;
					if (!LastTmH.IsKey(UrlSig) || LastTmH.GetDat(UrlSig) < Tm) 
						LastTmH.AddDat(UrlSig) = Tm;
				}
				printf(" 0 : Complete %d file out of %d files\n", ++fileCnt, filelist.Len());
			}
		}

		//printf("Total Url = %d\n Unique Url = %d\n", totUrl, FirstTmH.Len()); return; //TEST
		TFOut TmOut("InspectTm.bin");
		FirstTmH.Save(TmOut); LastTmH.Save(TmOut);

		// Second pass
		THash<TMd5Sig, TTriple<THashSet<TMd5Sig>, THashSet<TMd5Sig>, THashSet<TMd5Sig> > > FirstRec(RecSize, true), LastRec(RecSize, true);
	/*	
		fileCnt = 0;
		#pragma omp parallel for schedule(dynamic)
		for (int i = 0; i < filelist.Len(); i++) {
			if (FirstRec.Len() + LastRec.Len() > Mega(2)) continue;
			TMemesDataLoader Memes;
			Memes.LoadFile(filelist[i]);
			THashSet<TMd5Sig> ElemSet;
			TVec<TMd5Sig> UrlSigV;
			TVec<TTriple<THashSet<TMd5Sig>, THashSet<TMd5Sig>, THashSet<TMd5Sig> > > InfoV;
			TVec<TSecTm> TmV;

			while (Memes.LoadNext4(ElemSet)) {    
				if (strstr(Memes.PostUrlStr.CStr(), "facebook.com") != NULL) continue;
				if (strstr(Memes.PostUrlStr.CStr(), "twitter.com") != NULL) continue;
				TMd5Sig UrlSig = TMd5Sig(Memes.PostUrlStr);
				if (Memes.PubTm != FirstTmH.GetDat(UrlSig) && Memes.PubTm != LastTmH.GetDat(UrlSig)) continue;
				if (FirstTmH.GetDat(UrlSig) == LastTmH.GetDat(UrlSig))	continue;
			
				THashSet<TMd5Sig> LinkV, MemeV;
				for (int m = 0; m < Memes.MemeV.Len(); m++)
					if (Memes.MemeV[m].CountCh('?') <= Memes.MemeV[m].Len() / 2) {
						TStr qtStr = Memes.MemeV[m]; 
						FQuoteFilter(qtStr);
						MemeV.AddKey(TMd5Sig(qtStr));
					}

				for (int l = 0; l < Memes.LinkV.Len(); l++) 
					LinkV.AddKey(TMd5Sig(Memes.LinkV[l]));

				UrlSigV.Add(UrlSig);
				InfoV.Add(TTriple<THashSet<TMd5Sig>, THashSet<TMd5Sig>, THashSet<TMd5Sig> >(ElemSet, LinkV, MemeV));
				TmV.Add(Memes.PubTm);
			}
			#pragma omp critical
			{
				printf("Start critical \t\t");
				for (int j = 0; j < UrlSigV.Len(); j++) {
					TMd5Sig& UrlSig = UrlSigV[j];
					TTriple<THashSet<TMd5Sig>, THashSet<TMd5Sig>, THashSet<TMd5Sig> >& Info = InfoV[j];
					TSecTm& Tm = TmV[j];
					if (Tm == FirstTmH.GetDat(UrlSig))
						FirstRec.AddDat(UrlSig, Info);
					else 
						LastRec.AddDat(UrlSig, Info);
				}
				printf("End critical \n");
				printf("1: Complete %d file out of %d files FirstSize = %d LastSize = %d\n", ++fileCnt, filelist.Len(), FirstRec.Len(), LastRec.Len());
			}
		
		}
		TFOut FOut("Inspect.bin");
		FirstTmH.Save(FOut); LastTmH.Save(FOut); FirstRec.Save(FOut); LastRec.Save(FOut);
	*/
		TFIn FIn("Inspect.bin");
		FirstTmH.Load(FIn); LastTmH.Load(FIn); FirstRec.Load(FIn); LastRec.Load(FIn);

		double Intvl = 0.01;
		TIntH SimCntH;
		TIntH CtDifCntH;
		TIntH QtDifCntH;
		TIntH LkDifCntH;

		FILE* FSim = fopen("Sim.txt", "wt");
		for (int i = 0; i < LastRec.Len(); i++) {
			TMd5Sig UrlSig = LastRec.GetKey(i);
			if (!FirstRec.IsKey(UrlSig)) continue;
			if (LastTmH.GetDat(UrlSig).GetAbsSecs() - FirstTmH.GetDat(UrlSig).GetAbsSecs() < 7 * 24 * 3600) continue;
			TTriple<THashSet<TMd5Sig>, THashSet<TMd5Sig>, THashSet<TMd5Sig> >& FirstEntry = FirstRec.GetDat(UrlSig);
			TTriple<THashSet<TMd5Sig>, THashSet<TMd5Sig>, THashSet<TMd5Sig> >& LastEntry = LastRec[i];

			double CSim = ComputeSim(FirstEntry.Val1, LastEntry.Val1);
			double LSim = ComputeSim(FirstEntry.Val2, LastEntry.Val2);
			double QSim = ComputeSim(FirstEntry.Val3, LastEntry.Val3);
			//double Sim = sqrt(CSim*CSim + LSim*LSim + QSim*QSim) / sqrt(3.0);
			double Sim = CSim;
			fprintf(FSim,"%lf\t%lf\t%lf\t\n", CSim, LSim, QSim);
			SimCntH.AddDat(int(floor(Sim/Intvl))) += 1;

			if (Sim < 0.5) continue;
			CtDifCntH.AddDat((LastEntry.Val1.Len() - FirstEntry.Val1.Len())/20) += 1;
			if (LSim >= 0.5) QtDifCntH.AddDat(LastEntry.Val2.Len() - FirstEntry.Val2.Len()) += 1;
			if (QSim >= 0.5) LkDifCntH.AddDat(LastEntry.Val3.Len() - FirstEntry.Val3.Len()) += 1;
		}
		CtDifCntH.AddDat(0) = 0; QtDifCntH.AddDat(0) = 0; LkDifCntH.AddDat(0) = 0;
		ConstrainRange(CtDifCntH, -100, 100);
		ConstrainRange(QtDifCntH, -50, 50);
		ConstrainRange(LkDifCntH, -20, 20);

		int cnt = 0;
		for (int i = 0; i < SimCntH.Len(); i++) 
			cnt += SimCntH[i];
		printf("Total Count is %d\n", cnt);

		TGnuPlot::PlotValCntH(CtDifCntH, "SimCt", "Content Difference Plot", "# Difference unit = 20 word pairs", "Count");
		TGnuPlot::PlotValCntH(QtDifCntH, "SimQt", "Quote Difference Plot", "# of Difference in quotes", "Count");
		TGnuPlot::PlotValCntH(LkDifCntH, "SimLk", "Link Difference Plot", "# of Difference in links", "Count");
		TGnuPlot::PlotValCntH(SimCntH, "SimPlot", "Similarity Plot", "Similarity", "Count");
		printf("ALL DONE\n");
	}
#pragma endregion inspect
	
	printf("Dump Stats\n");

	// Dump Stats
	for (int i = 0; i < StatsH.Len(); i++) {
		printf("%s : %d\n", StatsH.GetKey(i).CStr(), StatsH[i].Val);
	}
}

void ConstrainRange(TIntH& CntH, int MnKey, int MxKey) {
	TIntV DelV;
	for (int i = 0; i < CntH.Len(); i++)
		if (CntH.GetKey(i) < MnKey || CntH.GetKey(i) > MxKey) 
			DelV.Add(CntH.GetKey(i));
	for (int i = 0; i < DelV.Len(); i++)
		CntH.DelKey(DelV[i]);
}

double ComputeSim(THashSet<TMd5Sig>& FirstSet, THashSet<TMd5Sig>& LastSet) {
	double cnt = 0;
	for (int j = 0; j < FirstSet.Len(); j++) {
		if (LastSet.IsKey(FirstSet[j]))
			cnt ++;
	}
	if (LastSet.Len() == 0) 
		return 0;
	else
		return cnt / (FirstSet.Len() + LastSet.Len() - cnt);
}
